<!DOCTYPE HTML>
<html lang="zh-CN">


<head>
    <meta charset="utf-8">
    <meta name="keywords" content="第1章-安装K8S1.25高可用集群.md, 博客，个人经验分离，kubernetes,k8s,Linux运维，Python">
    <meta name="description" content="
kuberadm安装k8s1.25高可用集群

k8s环境规划：
 podSubnet（pod网段） 10.244.0.0&amp;#x2F;16
 serviceSubnet（service网段）: 10.96.0.0&amp;#x2F;12



K">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
    <meta name="renderer" content="webkit|ie-stand|ie-comp">
    <meta name="mobile-web-app-capable" content="yes">
    <meta name="format-detection" content="telephone=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
    <meta name="referrer" content="no-referrer-when-downgrade">
    <!-- Global site tag (gtag.js) - Google Analytics -->


    <title>第1章-安装K8S1.25高可用集群.md | 躺平的人生</title>
    <link rel="icon" type="image/png" href="/favicon.png">
    


    <!-- bg-cover style     -->



<link rel="stylesheet" type="text/css" href="/libs/awesome/css/all.min.css">
<link rel="stylesheet" type="text/css" href="/libs/materialize/materialize.min.css">
<link rel="stylesheet" type="text/css" href="/libs/aos/aos.css">
<link rel="stylesheet" type="text/css" href="/libs/animate/animate.min.css">
<link rel="stylesheet" type="text/css" href="/libs/lightGallery/css/lightgallery.min.css">
<link rel="stylesheet" type="text/css" href="/css/matery.css">
<link rel="stylesheet" type="text/css" href="/css/my.css">
<link rel="stylesheet" type="text/css" href="/css/dark.css" media="none" onload="if(media!='all')media='all'">




    <link rel="stylesheet" href="/libs/tocbot/tocbot.css">
    <link rel="stylesheet" href="/css/post.css">




    
        <link rel="stylesheet" type="text/css" href="/css/reward.css">
    



    <script src="/libs/jquery/jquery-3.6.0.min.js"></script>

<meta name="generator" content="Hexo 6.3.0"></head>


<body>
    <header class="navbar-fixed">
    <nav id="headNav" class="bg-color nav-transparent">
        <div id="navContainer" class="nav-wrapper container">
            <div class="brand-logo">
                <a href="/" class="waves-effect waves-light">
                    
                    <img src="/medias/logo.png" class="logo-img" alt="LOGO">
                    
                    <span class="logo-span">躺平的人生</span>
                </a>
            </div>
            

<a href="#" data-target="mobile-nav" class="sidenav-trigger button-collapse"><i class="fas fa-bars"></i></a>
<ul class="right nav-menu">
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/" class="waves-effect waves-light">
      
      <i class="fas fa-home" style="zoom: 0.6;"></i>
      
      <span>首页</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/tags" class="waves-effect waves-light">
      
      <i class="fas fa-tags" style="zoom: 0.6;"></i>
      
      <span>标签</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/categories" class="waves-effect waves-light">
      
      <i class="fas fa-bookmark" style="zoom: 0.6;"></i>
      
      <span>分类</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/archives" class="waves-effect waves-light">
      
      <i class="fas fa-archive" style="zoom: 0.6;"></i>
      
      <span>归档</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/about" class="waves-effect waves-light">
      
      <i class="fas fa-user-circle" style="zoom: 0.6;"></i>
      
      <span>关于</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/friends" class="waves-effect waves-light">
      
      <i class="fas fa-address-book" style="zoom: 0.6;"></i>
      
      <span>友情链接</span>
    </a>
    
  </li>
  
  <li>
    <a href="#searchModal" class="modal-trigger waves-effect waves-light">
      <i id="searchIcon" class="fas fa-search" title="搜索" style="zoom: 0.85;"></i>
    </a>
  </li>
  <li>
    <a href="javascript:;" class="waves-effect waves-light" onclick="switchNightMode()" title="深色/浅色模式" >
      <i id="sum-moon-icon" class="fas fa-sun" style="zoom: 0.85;"></i>
    </a>
  </li>
</ul>


<div id="mobile-nav" class="side-nav sidenav">

    <div class="mobile-head bg-color">
        
        <img src="/medias/logo.png" class="logo-img circle responsive-img">
        
        <div class="logo-name">躺平的人生</div>
        <div class="logo-desc">
            
            本站记录本人各种学习的旅途，笔记，用于巩固自我并启发后来人
            
        </div>
    </div>

    <ul class="menu-list mobile-menu-list">
        
        <li class="m-nav-item">
	  
		<a href="/" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-home"></i>
			
			首页
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/tags" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-tags"></i>
			
			标签
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/categories" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-bookmark"></i>
			
			分类
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/archives" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-archive"></i>
			
			归档
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/about" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-user-circle"></i>
			
			关于
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/friends" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-address-book"></i>
			
			友情链接
		</a>
          
        </li>
        
        
        <li><div class="divider"></div></li>
        <li>
            <a href="https://miaohsukang.gitee.io" class="waves-effect waves-light" target="_blank">
                <i class="fab fa-github-square fa-fw"></i>Fork Me
            </a>
        </li>
        
    </ul>
</div>


        </div>

        
            <style>
    .nav-transparent .github-corner {
        display: none !important;
    }

    .github-corner {
        position: absolute;
        z-index: 10;
        top: 0;
        right: 0;
        border: 0;
        transform: scale(1.1);
    }

    .github-corner svg {
        color: #F062A7;
        fill: #fff;
        height: 64px;
        width: 64px;
    }

    .github-corner:hover .octo-arm {
        animation: a 0.56s ease-in-out;
    }

    .github-corner .octo-arm {
        animation: none;
    }

    @keyframes a {
        0%,
        to {
            transform: rotate(0);
        }
        20%,
        60% {
            transform: rotate(-25deg);
        }
        40%,
        80% {
            transform: rotate(10deg);
        }
    }
</style>

<a href="https://miaohsukang.gitee.io" class="github-corner tooltipped hide-on-med-and-down" target="_blank"
   data-tooltip="Fork Me" data-position="left" data-delay="50">
    <svg viewBox="0 0 250 250" aria-hidden="true">
        <path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path>
        <path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2"
              fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path>
        <path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z"
              fill="currentColor" class="octo-body"></path>
    </svg>
</a>
        
    </nav>

</header>

    
<script src="/libs/cryptojs/crypto-js.min.js"></script>
<script>
    (function() {
        let pwd = '';
        if (pwd && pwd.length > 0) {
            if (pwd !== CryptoJS.SHA256(prompt('请输入访问本文章的密码')).toString(CryptoJS.enc.Hex)) {
                alert('密码错误，将返回主页！');
                location.href = '/';
            }
        }
    })();
</script>




<div class="bg-cover pd-header post-cover" style="background-image: url('https://www.kubernetes.org.cn/img/2020/03/external-etcd.png')">
    <div class="container" style="right: 0px;left: 0px;">
        <div class="row">
            <div class="col s12 m12 l12">
                <div class="brand">
                    <h1 class="description center-align post-title">第1章-安装K8S1.25高可用集群.md</h1>
                </div>
            </div>
        </div>
    </div>
</div>




<main class="post-container content">

    
    <div class="row">
    <div id="main-content" class="col s12 m12 l9">
        <!-- 文章内容详情 -->
<div id="artDetail">
    <div class="card">
        <div class="card-content article-info">
            <div class="row tag-cate">
                <div class="col s7">
                    
                    <div class="article-tag">
                        
                            <a href="/tags/k8s%E9%9B%86%E7%BE%A4%E5%AE%89%E8%A3%85/">
                                <span class="chip bg-color">k8s集群安装</span>
                            </a>
                        
                    </div>
                    
                </div>
                <div class="col s5 right-align">
                    
                    <div class="post-cate">
                        <i class="fas fa-bookmark fa-fw icon-category"></i>
                        
                            <a href="/categories/K8S%E7%AC%94%E8%AE%B0/" class="post-category">
                                K8S笔记
                            </a>
                        
                    </div>
                    
                </div>
            </div>

            <div class="post-info">
                
                <div class="post-date info-break-policy">
                    <i class="far fa-calendar-minus fa-fw"></i>发布日期:&nbsp;&nbsp;
                    2023-02-12
                </div>
                

                
                <div class="post-date info-break-policy">
                    <i class="far fa-calendar-check fa-fw"></i>更新日期:&nbsp;&nbsp;
                    2023-02-12
                </div>
                

                
                <div class="info-break-policy">
                    <i class="far fa-file-word fa-fw"></i>文章字数:&nbsp;&nbsp;
                    5.9k
                </div>
                

                
                <div class="info-break-policy">
                    <i class="far fa-clock fa-fw"></i>阅读时长:&nbsp;&nbsp;
                    27 分
                </div>
                

                
                    <div id="busuanzi_container_page_pv" class="info-break-policy">
                        <i class="far fa-eye fa-fw"></i>阅读次数:&nbsp;&nbsp;
                        <span id="busuanzi_value_page_pv"></span>
                    </div>
				
            </div>
        </div>
        <hr class="clearfix">

        
        <!-- 是否加载使用自带的 prismjs. -->
        <link rel="stylesheet" href="/libs/prism/prism.min.css">
        

        

        <div class="card-content article-card-content">
            <div id="articleContent">
                <blockquote>
<p><strong>kuberadm安装k8s1.25高可用集群</strong></p>
</blockquote>
<p>k8s环境规划：</p>
<p> podSubnet（pod网段） 10.244.0.0&#x2F;16</p>
<p> serviceSubnet（service网段）: 10.96.0.0&#x2F;12</p>
<table>
<thead>
<tr>
<th>K8S集群角色</th>
<th>IP</th>
<th>主机名</th>
<th>安装的组件</th>
</tr>
</thead>
<tbody><tr>
<td>控制节点</td>
<td>10.10.30.180</td>
<td>master1</td>
<td>apiserver、controller-manager、schedule、kubelet、etcd、kube-proxy、容器运行时、calico、keepalived、nginx</td>
</tr>
<tr>
<td>工作节点</td>
<td>10.10.30.181</td>
<td>node1</td>
<td>Kube-proxy、calico、coredns、容器运行时、kubelet</td>
</tr>
<tr>
<td>工作节点</td>
<td>10.10.30.182</td>
<td>node2</td>
<td>Kube-proxy、calico、coredns、容器运行时、kubelet</td>
</tr>
</tbody></table>
<h3 id="一、初始化安装k8s集群环境"><a href="#一、初始化安装k8s集群环境" class="headerlink" title="一、初始化安装k8s集群环境"></a>一、初始化安装k8s集群环境</h3><h4 id="1-1-修改机器IP为静态IP"><a href="#1-1-修改机器IP为静态IP" class="headerlink" title="1.1 修改机器IP为静态IP"></a>1.1 修改机器IP为静态IP</h4><pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token function">vim</span> /etc/sysconfig/network-scripts/ifcfg-ens33文件
<span class="token assign-left variable">TYPE</span><span class="token operator">=</span>Ethernet
<span class="token assign-left variable">PROXY_METHOD</span><span class="token operator">=</span>none
<span class="token assign-left variable">BROWSER_ONLY</span><span class="token operator">=</span>no
<span class="token assign-left variable">BOOTPROTO</span><span class="token operator">=</span>static
<span class="token assign-left variable">IPADDR</span><span class="token operator">=</span><span class="token number">10.10</span>.30.180
<span class="token assign-left variable">NETMASK</span><span class="token operator">=</span><span class="token number">255.255</span>.255.0
<span class="token assign-left variable">GATEWAY</span><span class="token operator">=</span><span class="token number">10.10</span>.30.2
<span class="token assign-left variable">DNS1</span><span class="token operator">=</span><span class="token number">10.10</span>.30.2
<span class="token assign-left variable">DEFROUTE</span><span class="token operator">=</span>yes
<span class="token assign-left variable">IPV4_FAILURE_FATAL</span><span class="token operator">=</span>no
<span class="token assign-left variable">IPV6INIT</span><span class="token operator">=</span>yes
<span class="token assign-left variable">IPV6_AUTOCONF</span><span class="token operator">=</span>yes
<span class="token assign-left variable">IPV6_DEFROUTE</span><span class="token operator">=</span>yes
<span class="token assign-left variable">IPV6_FAILURE_FATAL</span><span class="token operator">=</span>no
<span class="token assign-left variable">IPV6_ADDR_GEN_MODE</span><span class="token operator">=</span>stable-privacy
<span class="token assign-left variable">NAME</span><span class="token operator">=</span>ens33
<span class="token assign-left variable">DEVICE</span><span class="token operator">=</span>ens33
<span class="token assign-left variable">ONBOOT</span><span class="token operator">=</span>yes<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>#修改配置文件之后需要重启网络服务才能使配置生效，重启网络服务命令如下：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash">systemctl restart network<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<h4 id="1-2-关闭selinux，所有k8s机器均操作"><a href="#1-2-关闭selinux，所有k8s机器均操作" class="headerlink" title="1.2 关闭selinux，所有k8s机器均操作"></a>1.2 关闭selinux，所有k8s机器均操作</h4><pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token function">sed</span> <span class="token parameter variable">-i</span> <span class="token string">'s/SELINUX=enforcing/SELINUX=disabled/g'</span> /etc/selinux/config<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>修改selinux配置文件之后，重启机器，selinux配置才能永久生效，重启之后，登录到机器，执行如下命令：</p>
<p>getenforce  # 如果显示Disabled说明selinux已经关闭</p>
<h4 id="1-3-配置机器主机名"><a href="#1-3-配置机器主机名" class="headerlink" title="1.3 配置机器主机名"></a>1.3 配置机器主机名</h4><p>在10.10.30.180上执行如下：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash">hostnamectl set-hostname master1 <span class="token operator">&amp;&amp;</span> <span class="token function">bash</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>在10.10.30.181上执行如下：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash">hostnamectl set-hostname node1 <span class="token operator">&amp;&amp;</span> <span class="token function">bash</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>在10.10.30.182上执行如下：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash">hostnamectl set-hostname node2 <span class="token operator">&amp;&amp;</span> <span class="token function">bash</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<h4 id="1-4-配置host文件，配置互信"><a href="#1-4-配置host文件，配置互信" class="headerlink" title="1.4 配置host文件，配置互信"></a>1.4 配置host文件，配置互信</h4><p>修改三台机器的&#x2F;etc&#x2F;&#x2F;hosts 文件，增加如下内容：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># cat /etc/hosts</span>
<span class="token number">127.0</span>.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
<span class="token number">10.10</span>.30.180  master1
<span class="token number">10.10</span>.30.181  node1
<span class="token number">10.10</span>.30.182  node2<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>互信配置如下：<br>配置master1到其他机器免密登录<br>[root@master1 ~]# ssh-keygen       # 一路回车，不输入密码<br>把本地生成的密钥文件和私钥文件拷贝到远程主机</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id master1</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id node1</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id node2 </span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>配置node1到其他机器免密登录<br>[root@node1~]# ssh-keygen  # 一路回车，不输入密码<br>把本地生成的密钥文件和私钥文件拷贝到远程主机</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id master1</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id node1</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id node2</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>配置node2到其他机器免密登录<br>[root@ node2~]# ssh-keygen  # 一路回车，不输入密码<br>把本地生成的密钥文件和私钥文件拷贝到远程主机</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@ node2~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id master1</span>
<span class="token punctuation">[</span>root@ node2~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id node1</span>
<span class="token punctuation">[</span>root@ node2~<span class="token punctuation">]</span><span class="token comment"># ssh-copy-id node2</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<h4 id="1-5-关闭交换分区swap-提升性能"><a href="#1-5-关闭交换分区swap-提升性能" class="headerlink" title="1.5 关闭交换分区swap,提升性能"></a>1.5 关闭交换分区swap,提升性能</h4><p>临时关闭</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># swapoff -a</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># swapoff -a</span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># swapoff -a</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>永久关闭：注释swap挂载，给swap这行开头加一下注释</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># vim /etc/fstab  </span>
<span class="token comment">#/dev/mapper/centos-swap swap    swap   defaults     0 0</span>

<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># vim /etc/fstab</span>
<span class="token comment">#/dev/mapper/centos-swap swap    swap   defaults     0 0</span>

<span class="token punctuation">[</span>root@node2~<span class="token punctuation">]</span><span class="token comment"># vim /etc/fstab</span>
<span class="token comment">#/dev/mapper/centos-swap swap    swap   defaults     0 0</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p><strong>问题1</strong>：为什么要关闭swap交换分区？<br>Swap是交换分区，如果机器内存不够，会使用swap分区，但是swap分区的性能较低，k8s设计的时候为了能提升性能，默认是不允许使用交换分区的。Kubeadm初始化的时候会检测swap是否关闭，如果没关闭，那就初始化失败。如果不想要关闭交换分区，安装k8s的时候可以指定–ignore-preflight-errors&#x3D;Swap来解决。</p>
<h4 id="1-6-修改机器内核参数"><a href="#1-6-修改机器内核参数" class="headerlink" title="1.6 修改机器内核参数"></a>1.6 修改机器内核参数</h4><pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># modprobe br_netfilter</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># cat > /etc/sysctl.d/k8s.conf &lt;&lt;EOF</span>
net.bridge.bridge-nf-call-ip6tables <span class="token operator">=</span> <span class="token number">1</span>
net.bridge.bridge-nf-call-iptables <span class="token operator">=</span> <span class="token number">1</span>
net.ipv4.ip_forward <span class="token operator">=</span> <span class="token number">1</span>
EOF
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># sysctl -p /etc/sysctl.d/k8s.conf</span>

<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># modprobe br_netfilter</span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># cat > /etc/sysctl.d/k8s.conf &lt;&lt;EOF</span>
net.bridge.bridge-nf-call-ip6tables <span class="token operator">=</span> <span class="token number">1</span>
net.bridge.bridge-nf-call-iptables <span class="token operator">=</span> <span class="token number">1</span>
net.ipv4.ip_forward <span class="token operator">=</span> <span class="token number">1</span>
EOF

<span class="token punctuation">[</span>root@node1~<span class="token punctuation">]</span><span class="token comment"># sysctl -p /etc/sysctl.d/k8s.conf</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># modprobe br_netfilter</span>
<span class="token punctuation">[</span>root@node1~<span class="token punctuation">]</span><span class="token comment"># cat > /etc/sysctl.d/k8s.conf &lt;&lt;EOF</span>
net.bridge.bridge-nf-call-ip6tables <span class="token operator">=</span> <span class="token number">1</span>
net.bridge.bridge-nf-call-iptables <span class="token operator">=</span> <span class="token number">1</span>
net.ipv4.ip_forward <span class="token operator">=</span> <span class="token number">1</span>
EOF
<span class="token punctuation">[</span>root@ node1~<span class="token punctuation">]</span><span class="token comment"># sysctl -p /etc/sysctl.d/k8s.conf</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p><strong>问题1</strong>：sysctl是做什么的？</p>
<p>在运行时配置内核参数<br> -p  从指定的文件加载系统参数，如不指定即从&#x2F;etc&#x2F;sysctl.conf中加载</p>
<p><strong>sysctl -p &#x2F;etc&#x2F;sysctl.d&#x2F;k8s.conf出现报错：</strong></p>
 <pre class="line-numbers language-bash" data-language="bash"><code class="language-bash">sysctl: cannot <span class="token function">stat</span> /proc/sys/net/bridge/bridge-nf-call-ip6tables: No such <span class="token function">file</span> or directory
sysctl: cannot <span class="token function">stat</span> /proc/sys/net/bridge/bridge-nf-call-iptables: No such <span class="token function">file</span> or directory<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>解决方法：<br>modprobe br_netfilter</p>
<p><strong>问题2</strong>：为什么开启net.bridge.bridge-nf-call-iptables内核参数？</p>
<p>在centos下安装docker，执行docker info出现如下警告：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash">WARNING: bridge-nf-call-iptables is disabled
WARNING: bridge-nf-call-ip6tables is disabled<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>解决办法：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token function">vim</span> /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables <span class="token operator">=</span> <span class="token number">1</span>
net.bridge.bridge-nf-call-iptables <span class="token operator">=</span> <span class="token number">1</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>问题3：为什么要开启net.ipv4.ip_forward &#x3D; 1参数？</p>
<p>kubeadm初始化k8s如果报错,就表示没有开启ip_forward，需要开启。</p>
<blockquote>
<p>net.ipv4.ip_forward是数据包转发：</p>
<p>出于安全考虑，Linux系统默认是禁止数据包转发的。所谓转发即当主机拥有多于一块的网卡时，其中一块收到数据包，根据数据包的目的ip地址将数据包发往本机另一块网卡，该网卡根据路由表继续发送数据包。这通常是路由器所要实现的功能。</p>
<p>要让Linux系统具有路由转发功能，需要配置一个Linux的内核参数net.ipv4.ip_forward。这个参数指定了Linux系统当前对路由转发功能的支持情况；其值为0时表示禁止进行IP转发；如果是1,则说明IP转发功能已经打开。</p>
</blockquote>
<h4 id="1-7-关闭防火墙"><a href="#1-7-关闭防火墙" class="headerlink" title="1.7 关闭防火墙"></a>1.7 关闭防火墙</h4><pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># systemctl stop firewalld ; systemctl disable firewalld</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># systemctl stop firewalld ; systemctl disable firewalld</span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># systemctl stop firewalld ; systemctl disable firewalld</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<h4 id="1-9-配置阿里云的repo源"><a href="#1-9-配置阿里云的repo源" class="headerlink" title="1.9 配置阿里云的repo源"></a>1.9 配置阿里云的repo源</h4><p>配置国内安装docker和containerd的阿里云的repo源，<strong>三台</strong>机器都执行如下两条命令：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash">yum <span class="token function">install</span> yum-utils <span class="token parameter variable">-y</span>
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<h4 id="1-10-配置安装K8S组件需要的阿里云repo源"><a href="#1-10-配置安装K8S组件需要的阿里云repo源" class="headerlink" title="1.10 配置安装K8S组件需要的阿里云repo源"></a>1.10 配置安装K8S组件需要的阿里云repo源</h4><p>三台机器都要执行：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token function">cat</span> <span class="token operator">></span> /etc/yum.repos.d/kubernetes.repo <span class="token operator">&lt;&lt;</span><span class="token string">EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
EOF</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h4 id="1-11-配置时间同步"><a href="#1-11-配置时间同步" class="headerlink" title="1.11 配置时间同步"></a>1.11 配置时间同步</h4><pre class="line-numbers language-bash" data-language="bash"><code class="language-bash">在master1上执行如下：
<span class="token comment">#安装ntpdate命令</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># yum install ntpdate -y</span>
<span class="token comment">#跟网络时间做同步</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># ntpdate cn.pool.ntp.org</span>
<span class="token comment">#把时间同步做成计划任务</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># crontab -e</span>
* */1 * * * /usr/sbin/ntpdate   cn.pool.ntp.org
<span class="token comment">#重启crond服务</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#service crond restart</span>

其它两台机器node1 node2参考master执行<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h4 id="1-12-安装基础软件包"><a href="#1-12-安装基础软件包" class="headerlink" title="1.12 安装基础软件包"></a>1.12 安装基础软件包</h4><p>三台服务器都要执行</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># yum install -y device-mapper-persistent-data lvm2 wget net-tools nfs-utils lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo ntp libaio-devel wget vim ncurses-devel autoconf automake zlib-devel? python-devel epel-release openssh-server socat? ipvsadm conntrack telnet ipvsadm</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<h4 id="1-13-安装containerd服务"><a href="#1-13-安装containerd服务" class="headerlink" title="1.13 安装containerd服务"></a>1.13 安装containerd服务</h4><h5 id="1-13-1-安装containerd"><a href="#1-13-1-安装containerd" class="headerlink" title="1.13.1 安装containerd"></a>1.13.1 安装containerd</h5><pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#yum install containerd.io-1.6.6 -y</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>接下来生成 containerd 的配置文件:</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#mkdir -p /etc/containerd</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#containerd config default > /etc/containerd/config.toml</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<h5 id="1-13-2-修改配置文件"><a href="#1-13-2-修改配置文件" class="headerlink" title="1.13.2 修改配置文件"></a>1.13.2 修改配置文件</h5><p>打开&#x2F;etc&#x2F;containerd&#x2F;config.toml<br>把SystemdCgroup &#x3D; <strong>false</strong>修改成SystemdCgroup &#x3D; <strong>true</strong></p>
<p>把sandbox_image &#x3D; <strong>“k8s.gcr.io&#x2F;pause:3.6”<strong>修改成sandbox_image&#x3D;</strong>“registry.aliyuncs.com&#x2F;google_containers&#x2F;pause:3.7”</strong></p>
<p>修改完成可以检查一下：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># egrep -i "systemdc|sandbox"  /etc/containerd/config.toml </span>
    sandbox_image <span class="token operator">=</span> <span class="token string">"registry.aliyuncs.com/google_containers/pause:3.7"</span>
            SystemdCgroup <span class="token operator">=</span> <span class="token boolean">true</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p> 配置 containerd 开机启动，并启动 containerd</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#systemctl enable containerd  --now</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>上面1.13.1和1.13.2在剩下两台机器node1  node2上面也同步执行</p>
<h5 id="1-13-3-修改crictl文件"><a href="#1-13-3-修改crictl文件" class="headerlink" title="1.13.3 修改crictl文件"></a>1.13.3 修改crictl文件</h5><p>crictl命令用来查看k8s中的一些镜像，所以需要修改一下&#x2F;etc&#x2F;crictl.yaml文件</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#cat > /etc/crictl.yaml &lt;&lt;EOF</span>
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: <span class="token number">10</span>
debug: <span class="token boolean">false</span>
EOF

<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#systemctl restart containerd</span>

<span class="token comment"># 其它机器同步执行以上操作</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>备注：docker也要安装，docker跟containerd不冲突，安装docker是为了能基于dockerfile构建镜像</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#yum install  docker-ce  -y</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment">#yum install  docker-ce  -y</span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment">#yum install  docker-ce  -y</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment">#systemctl enable docker --now</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment">#systemctl enable docker --now</span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment">#systemctl enable docker --now</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>配置containerd镜像加速器，k8s所有节点均按照以下配置：<br>编辑vim &#x2F;etc&#x2F;containerd&#x2F;config.toml文件<br>找到config_path &#x3D; “”，修改成如下目录：<br><code>config_path = &quot;/etc/containerd/certs.d&quot;</code><br>修改完成后，保存退出，然后去创建这个目录<br>mkdir &#x2F;etc&#x2F;containerd&#x2F;certs.d&#x2F;docker.io&#x2F; -p</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token function">vim</span> /etc/containerd/certs.d/docker.io/hosts.toml
<span class="token comment"># 写入如下内容：</span>
<span class="token punctuation">[</span>host.<span class="token string">"https://vh3bm52y.mirror.aliyuncs.com"</span>,host.<span class="token string">"https://registry.docker-cn.com"</span><span class="token punctuation">]</span>
  capabilities <span class="token operator">=</span> <span class="token punctuation">[</span><span class="token string">"pull"</span><span class="token punctuation">]</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>

<p>重启containerd：<br><code>systemctl restart containerd</code></p>
<p>配置docker镜像加速器，k8s所有节点均按照以下配置:<br>vim &#x2F;etc&#x2F;docker&#x2F;daemon.json<br>写入如下内容：</p>
<pre class="line-numbers language-json" data-language="json"><code class="language-json"><span class="token punctuation">&#123;</span>
 <span class="token property">"registry-mirrors"</span><span class="token operator">:</span><span class="token punctuation">[</span><span class="token string">"https://vh3bm52y.mirror.aliyuncs.com"</span><span class="token punctuation">,</span><span class="token string">"https://registry.docker-cn.com"</span><span class="token punctuation">,</span><span class="token string">"https://docker.mirrors.ustc.edu.cn"</span><span class="token punctuation">,</span><span class="token string">"https://dockerhub.azk8s.cn"</span><span class="token punctuation">,</span><span class="token string">"http://hub-mirror.c.163.com"</span><span class="token punctuation">]</span>
<span class="token punctuation">&#125;</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>重启docker：<br><code>systemctl restart docker</code></p>
<h4 id="1-14-安装初始化k8s需要的软件"><a href="#1-14-安装初始化k8s需要的软件" class="headerlink" title="1.14 安装初始化k8s需要的软件"></a>1.14 安装初始化k8s需要的软件</h4><pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># yum install -y kubelet-1.25.0 kubeadm-1.25.0 kubectl-1.25.0</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># systemctl enable kubelet</span>

<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># yum install -y kubelet-1.25.0 kubeadm-1.25.0 kubectl-1.25.0</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># systemctl enable kubelet</span>

<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># yum install -y kubelet-1.25.0 kubeadm-1.25.0 kubectl-1.25.0</span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># systemctl enable kubelet</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>注：每个软件包的作用</p>
<blockquote>
<p>Kubeadm: kubeadm是一个工具，用来初始化k8s集群的<br>kubelet: 安装在集群所有节点上，用于启动Pod的，kubeadm安装k8s，k8s控制节点和工作节点的组件，都是基于pod运行的，只要pod启动，就需要 kubelet<br>kubectl: 通过kubectl可以部署和管理应用，查看各种资源，创建、删除和更新各种组件</p>
</blockquote>
<h4 id="1-15-设置容器进行时"><a href="#1-15-设置容器进行时" class="headerlink" title="1.15 设置容器进行时"></a>1.15 设置容器进行时</h4><p>在三台节点都执行</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1~<span class="token punctuation">]</span><span class="token comment"># crictl config runtime-endpoint /run/containerd/containerd.sock</span>
<span class="token punctuation">[</span>root@node1~<span class="token punctuation">]</span><span class="token comment"># crictl config runtime-endpoint /run/containerd/containerd.sock</span>
<span class="token punctuation">[</span>root@node2~<span class="token punctuation">]</span><span class="token comment"># crictl config runtime-endpoint /run/containerd/containerd.sock</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<h4 id="1-16-生成kubeadm初始化文件"><a href="#1-16-生成kubeadm初始化文件" class="headerlink" title="1.16 生成kubeadm初始化文件"></a>1.16 生成kubeadm初始化文件</h4><p>生成kube初始化安装yaml文件，只在master1节点执行</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubeadm config print init-defaults > kubeadm.yaml</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>生成后，根据我们自己的需求修改配置，比如修改 imageRepository 的值，kube-proxy 的模式为 ipvs，需要注意的是由于我们使用的containerd作为运行时，所以在初始化节点的时候需要指定cgroupDriver为systemd</p>
<p>kubeadm.yaml配置文件如下：</p>
<pre class="line-numbers language-yaml" data-language="yaml"><code class="language-yaml"><span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> kubeadm.k8s.io/v1beta3
<span class="token key atrule">bootstrapTokens</span><span class="token punctuation">:</span>
<span class="token punctuation">-</span> <span class="token key atrule">groups</span><span class="token punctuation">:</span>
  <span class="token punctuation">-</span> system<span class="token punctuation">:</span>bootstrappers<span class="token punctuation">:</span>kubeadm<span class="token punctuation">:</span>default<span class="token punctuation">-</span>node<span class="token punctuation">-</span>token
  <span class="token key atrule">token</span><span class="token punctuation">:</span> abcdef.0123456789abcdef
  <span class="token key atrule">ttl</span><span class="token punctuation">:</span> 24h0m0s
  <span class="token key atrule">usages</span><span class="token punctuation">:</span>
  <span class="token punctuation">-</span> signing
  <span class="token punctuation">-</span> authentication
<span class="token key atrule">kind</span><span class="token punctuation">:</span> InitConfiguration
<span class="token key atrule">localAPIEndpoint</span><span class="token punctuation">:</span>
  <span class="token key atrule">advertiseAddress</span><span class="token punctuation">:</span> 10.10.30.180  <span class="token comment"># 控制节点的IP</span>
  <span class="token key atrule">bindPort</span><span class="token punctuation">:</span> <span class="token number">6443</span>
<span class="token key atrule">nodeRegistration</span><span class="token punctuation">:</span>
  <span class="token key atrule">criSocket</span><span class="token punctuation">:</span> unix<span class="token punctuation">:</span>///run/containerd/containerd.sock  <span class="token comment"># 指定容器运行时</span>
  <span class="token key atrule">imagePullPolicy</span><span class="token punctuation">:</span> IfNotPresent
  <span class="token key atrule">name</span><span class="token punctuation">:</span> master1  <span class="token comment"># 控制节点主机名</span>
  <span class="token key atrule">taints</span><span class="token punctuation">:</span> <span class="token null important">null</span>
<span class="token punctuation">---</span>
<span class="token key atrule">apiServer</span><span class="token punctuation">:</span>
  <span class="token key atrule">timeoutForControlPlane</span><span class="token punctuation">:</span> 4m0s
<span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> kubeadm.k8s.io/v1beta3
<span class="token key atrule">certificatesDir</span><span class="token punctuation">:</span> /etc/kubernetes/pki
<span class="token key atrule">clusterName</span><span class="token punctuation">:</span> kubernetes
<span class="token key atrule">controllerManager</span><span class="token punctuation">:</span> <span class="token punctuation">&#123;</span><span class="token punctuation">&#125;</span>
<span class="token key atrule">dns</span><span class="token punctuation">:</span> <span class="token punctuation">&#123;</span><span class="token punctuation">&#125;</span>
<span class="token key atrule">etcd</span><span class="token punctuation">:</span>
  <span class="token key atrule">local</span><span class="token punctuation">:</span>
    <span class="token key atrule">dataDir</span><span class="token punctuation">:</span> /var/lib/etcd
<span class="token key atrule">imageRepository</span><span class="token punctuation">:</span> registry.cn<span class="token punctuation">-</span>hangzhou.aliyuncs.com/google_containers
<span class="token key atrule">kind</span><span class="token punctuation">:</span> ClusterConfiguration  <span class="token comment"># 指定阿里云镜像仓库地址</span>
<span class="token key atrule">kubernetesVersion</span><span class="token punctuation">:</span> 1.25.0  <span class="token comment"># k8s版本号</span>
<span class="token key atrule">networking</span><span class="token punctuation">:</span>
  <span class="token key atrule">dnsDomain</span><span class="token punctuation">:</span> cluster.local
  <span class="token key atrule">serviceSubnet</span><span class="token punctuation">:</span> 10.96.0.0/12  <span class="token comment"># 指定service网段</span>
  <span class="token key atrule">podSubnet</span><span class="token punctuation">:</span> 10.224.0.0/16  <span class="token comment"># 增加行，指定pod网段</span>
<span class="token key atrule">scheduler</span><span class="token punctuation">:</span> <span class="token punctuation">&#123;</span><span class="token punctuation">&#125;</span> 
<span class="token comment"># 在文件最后，插入以下内容，复制时注意要带---</span>
<span class="token punctuation">---</span>
<span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> kubeproxy.config.k8s.io/v1alpha1
<span class="token key atrule">kind</span><span class="token punctuation">:</span> KubeProxyConfiguration
<span class="token key atrule">mode</span><span class="token punctuation">:</span> ipvs
<span class="token punctuation">---</span>
<span class="token key atrule">apiVersion</span><span class="token punctuation">:</span> kubelet.config.k8s.io/v1beta1
<span class="token key atrule">kind</span><span class="token punctuation">:</span> KubeletConfiguration
<span class="token key atrule">cgroupDriver</span><span class="token punctuation">:</span> systemd<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<blockquote>
<p>特别提醒：–image-repository  registry.aliyuncs.com&#x2F;google_containers为保证拉取镜像不到国外站点拉取，手动指定仓库地址为registry.aliyuncs.com&#x2F;google_containers。kubeadm默认从k8s.gcr.io拉取镜像  我们本地有导入到的离线镜像，所以会优先使用本地的镜像。</p>
<p>mode: ipvs 表示kube-proxy代理模式是ipvs，如果不指定ipvs，会默认使用iptables，但是iptables效率低，所以我们生产环境建议开启ipvs，阿里云和华为云托管的K8s，也提供ipvs模式</p>
</blockquote>
<h4 id="1-17-导入k8s镜像"><a href="#1-17-导入k8s镜像" class="headerlink" title="1.17 导入k8s镜像"></a>1.17 导入k8s镜像</h4><p>使用ctr命令指定命名空间导入镜像</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># ctr -n=k8s.io images import k8s_1.25.0.tar.gz</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># ctr -n=k8s.io images import k8s_1.25.0.tar.gz</span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># ctr -n=k8s.io images import k8s_1.25.0.tar.gz</span>

<span class="token comment"># 查看镜像：crictl images</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<blockquote>
<p>备注！这个k8s_1.25.0.tar.gz文件如何来的？<br>这个文件把安装k8s需要的镜像都集成好了，这个是我第一次安装1.25.0这个版本，获取到对应的镜像，通过ctr images export 这个命令把镜像输出到k8s_1.25.0.tar.gz文件，如果大家安装其他版本，那就不需要实现解压镜像，可以默认从网络拉取镜像即可。</p>
<blockquote>
<p>ctr是containerd自带的工具，有<strong>命名空间</strong>的概念，若是k8s相关的镜像，都默认在k8s.io这个命名空间，所以导入镜像时需要指定命令空间为k8s.io</p>
</blockquote>
</blockquote>
<h4 id="1-18-基于kubeadm-yaml初始化k8s集群"><a href="#1-18-基于kubeadm-yaml初始化k8s集群" class="headerlink" title="1.18 基于kubeadm.yaml初始化k8s集群"></a>1.18 基于kubeadm.yaml初始化k8s集群</h4><pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubeadm init --config=kubeadm.yaml --ignore-preflight-errors=SystemVerification</span><span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<p>显示如下结果，说明安装成功<br>![[Pasted image 20221101223158.png]]</p>
<p>配置kubectl的配置文件config，相当于对kubectl进行授权，这样kubectl命令可以使用这个证书对k8s集群进行管理</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># mkdir -p $HOME/.kube</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config</span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># sudo chown $(id -u):$(id -g) $HOME/.kube/config</span>

<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes</span>
NAME      STATUS   ROLES           AGE   VERSION
master1   Ready    control-plane   24m   v1.25.0<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h3 id="二、扩容k8s集群"><a href="#二、扩容k8s集群" class="headerlink" title="二、扩容k8s集群"></a>二、扩容k8s集群</h3><h4 id="2-1-添加第一个工作节点"><a href="#2-1-添加第一个工作节点" class="headerlink" title="2.1 添加第一个工作节点"></a>2.1 添加第一个工作节点</h4><p>在master1上查看加入节点的命令：</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubeadm token create --print-join-command</span>
kubeadm <span class="token function">join</span> <span class="token number">10.10</span>.30.180:6443 <span class="token parameter variable">--token</span> dgjher.cvk7i2lgn0gs323y --discovery-token-ca-cert-hash sha256:886a1cf6a95b4aa24ecb50e7b841664038fcd2c1fa956970b60d339039e83e62<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>将上面输出结果，得到到Node1节点执行</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># kubeadm join 10.10.30.180:6443 --token dgjher.cvk7i2lgn0gs323y --discovery-token-ca-cert-hash sha256:886a1cf6a95b4aa24ecb50e7b841664038fcd2c1fa956970b60d339039e83e62</span>
<span class="token punctuation">[</span>preflight<span class="token punctuation">]</span> Running pre-flight checks
<span class="token punctuation">[</span>preflight<span class="token punctuation">]</span> Reading configuration from the cluster<span class="token punctuation">..</span>.
<span class="token punctuation">[</span>preflight<span class="token punctuation">]</span> FYI: You can <span class="token function">look</span> at this config <span class="token function">file</span> with <span class="token string">'kubectl -n kube-system get cm kubeadm-config -o yaml'</span>
<span class="token punctuation">[</span>kubelet-start<span class="token punctuation">]</span> Writing kubelet configuration to <span class="token function">file</span> <span class="token string">"/var/lib/kubelet/config.yaml"</span>
<span class="token punctuation">[</span>kubelet-start<span class="token punctuation">]</span> Writing kubelet environment <span class="token function">file</span> with flags to <span class="token function">file</span> <span class="token string">"/var/lib/kubelet/kubeadm-flags.env"</span>
<span class="token punctuation">[</span>kubelet-start<span class="token punctuation">]</span> Starting the kubelet
<span class="token punctuation">[</span>kubelet-start<span class="token punctuation">]</span> Waiting <span class="token keyword">for</span> the kubelet to perform the TLS Bootstrap<span class="token punctuation">..</span>.

This <span class="token function">node</span> has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run <span class="token string">'kubectl get nodes'</span> on the control-plane to see this <span class="token function">node</span> <span class="token function">join</span> the cluster.<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>添加完成后，可以通过以下命令查看当前集群节点状态</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes</span>
NAME      STATUS   ROLES           AGE   VERSION
master1   Ready    control-plane   27m   v1.25.0
node1     Ready    <span class="token operator">&lt;</span>none<span class="token operator">></span>          59s   v1.25.0<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>

<h4 id="2-2-添加第二个工作节点"><a href="#2-2-添加第二个工作节点" class="headerlink" title="2.2 添加第二个工作节点"></a>2.2 添加第二个工作节点</h4><p>使用同样的方法添加node2节点到集群当中</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubeadm token create --print-join-command</span>
kubeadm <span class="token function">join</span> <span class="token number">10.10</span>.30.180:6443 <span class="token parameter variable">--token</span> ggnzvw.ov2niivc9kepx9ff --discovery-token-ca-cert-hash sha256:886a1cf6a95b4aa24ecb50e7b841664038fcd2c1fa956970b60d339039e83e62 <span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>将上面输出结果，得到到Node2节点执行</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># kubeadm join 10.10.30.180:6443 --token ggnzvw.ov2niivc9kepx9ff --discovery-token-ca-cert-hash sha256:886a1cf6a95b4aa24ecb50e7b841664038fcd2c1fa956970b60d339039e83e62 </span>
<span class="token punctuation">[</span>preflight<span class="token punctuation">]</span> Running pre-flight checks
<span class="token punctuation">[</span>preflight<span class="token punctuation">]</span> Reading configuration from the cluster<span class="token punctuation">..</span>.
<span class="token punctuation">[</span>preflight<span class="token punctuation">]</span> FYI: You can <span class="token function">look</span> at this config <span class="token function">file</span> with <span class="token string">'kubectl -n kube-system get cm kubeadm-config -o yaml'</span>
<span class="token punctuation">[</span>kubelet-start<span class="token punctuation">]</span> Writing kubelet configuration to <span class="token function">file</span> <span class="token string">"/var/lib/kubelet/config.yaml"</span>
<span class="token punctuation">[</span>kubelet-start<span class="token punctuation">]</span> Writing kubelet environment <span class="token function">file</span> with flags to <span class="token function">file</span> <span class="token string">"/var/lib/kubelet/kubeadm-flags.env"</span>
<span class="token punctuation">[</span>kubelet-start<span class="token punctuation">]</span> Starting the kubelet
<span class="token punctuation">[</span>kubelet-start<span class="token punctuation">]</span> Waiting <span class="token keyword">for</span> the kubelet to perform the TLS Bootstrap<span class="token punctuation">..</span>.

This <span class="token function">node</span> has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run <span class="token string">'kubectl get nodes'</span> on the control-plane to see this <span class="token function">node</span> <span class="token function">join</span> the cluster.<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>添加完成后，可以通过以下命令查看当前集群节点状态</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes</span>
NAME      STATUS   ROLES           AGE    VERSION
master1   Ready    control-plane   29m    v1.25.0
node1     Ready    <span class="token operator">&lt;</span>none<span class="token operator">></span>          3m9s   v1.25.0
node2     Ready    <span class="token operator">&lt;</span>none<span class="token operator">></span>          81s    v1.25.0<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h4 id="2-3-对工作节点打标签，方便识别"><a href="#2-3-对工作节点打标签，方便识别" class="headerlink" title="2.3 对工作节点打标签，方便识别"></a>2.3 对工作节点打标签，方便识别</h4><p>语法：<code>kubectl label nodes &lt;node-name&gt; &lt;label-key&gt;=&lt;label-value&gt; </code></p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl label nodes node2 node-role.kubernetes.io/work=work</span>
node/node2 labeled
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl label nodes node1 node-role.kubernetes.io/work=work</span>
node/node1 labeled
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes</span>
NAME      STATUS   ROLES           AGE     VERSION
master1   Ready    control-plane   31m     v1.25.0
node1     Ready    work            4m12s   v1.25.0
node2     Ready    work            2m24s   v1.25.0<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>删除一个Label，只需在命令行最后指定Label的key名并与一个减号相连即可：</p>
<p>语法：<code>kubectl label nodes kube-node label_name-</code></p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes</span>
NAME      STATUS   ROLES           AGE     VERSION
master1   Ready    control-plane   3d15h   v1.25.0
node1     Ready    work,work1      3d15h   v1.25.0
node2     Ready    work,work2      3d15h   v1.25.0
<span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl label nodes node2 node-role.kubernetes.io/work2-</span>
node/node2 unlabeled
<span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl label nodes node1 node-role.kubernetes.io/work1-</span>
node/node1 unlabeled
<span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes</span>
NAME      STATUS   ROLES           AGE     VERSION
master1   Ready    control-plane   3d15h   v1.25.0
node1     Ready    work            3d15h   v1.25.0
node2     Ready    work            3d15h   v1.25.0
<span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl label nodes node1 node-role.kubernetes.io/work-</span>
node/node1 unlabeled
<span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl label nodes node2 node-role.kubernetes.io/work-</span>
node/node2 unlabeled
<span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes</span>
NAME      STATUS   ROLES           AGE     VERSION
master1   Ready    control-plane   3d15h   v1.25.0
node1     Ready    <span class="token operator">&lt;</span>none<span class="token operator">></span>          3d15h   v1.25.0
node2     Ready    <span class="token operator">&lt;</span>none<span class="token operator">></span>          3d15h   v1.25.0<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>修改一个label:</p>
<p>增加方法的基础上使用–overwrite 参数</p>
<p><code>kubectl label nodes kube-node label_name=label_value --overwrite</code></p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl label nodes node1 worker01=work01</span>
error: <span class="token string">'worker01'</span> already has a value <span class="token punctuation">(</span>worker01<span class="token punctuation">)</span>, and <span class="token parameter variable">--overwrite</span> is <span class="token boolean">false</span>
<span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl label nodes node1 worker01=work01 --overwrite</span>
node/node1 labeled
<span class="token punctuation">[</span>root@master1 pod<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes --show-labels</span>
NAME      STATUS   ROLES           AGE     VERSION   LABELS
master1   Ready    control-plane   3d16h   v1.25.0   beta.kubernetes.io/arch<span class="token operator">=</span>amd64,beta.kubernetes.io/os<span class="token operator">=</span>linux,kubernetes.io/arch<span class="token operator">=</span>amd64,kubernetes.io/hostname<span class="token operator">=</span>master1,kubernetes.io/os<span class="token operator">=</span>linux,node-role.kubernetes.io/control-plane<span class="token operator">=</span>,node.kubernetes.io/exclude-from-external-load-balancers<span class="token operator">=</span>
node1     Ready    work1           3d15h   v1.25.0   beta.kubernetes.io/arch<span class="token operator">=</span>amd64,beta.kubernetes.io/os<span class="token operator">=</span>linux,kubernetes.io/arch<span class="token operator">=</span>amd64,kubernetes.io/hostname<span class="token operator">=</span>node1,kubernetes.io/os<span class="token operator">=</span>linux,node-role.kubernetes.io/work1<span class="token operator">=</span>work1,worker01<span class="token operator">=</span>work01
node2     Ready    work2           3d15h   v1.25.0   beta.kubernetes.io/arch<span class="token operator">=</span>amd64,beta.kubernetes.io/os<span class="token operator">=</span>linux,kubernetes.io/arch<span class="token operator">=</span>amd64,kubernetes.io/hostname<span class="token operator">=</span>node2,kubernetes.io/os<span class="token operator">=</span>linux,node-role.kubernetes.io/work2<span class="token operator">=</span>work2,worker02<span class="token operator">=</span>worker02<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h4 id="2-4-安装kubernetes网络插件-Calico"><a href="#2-4-安装kubernetes网络插件-Calico" class="headerlink" title="2.4 安装kubernetes网络插件-Calico"></a>2.4 安装kubernetes网络插件-Calico</h4><p>把安装calico需要的镜像calico.tar.gz传到master1、xianchaomaster2、xianchaomaster3和node1节点，手动解压</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># ctr -n=k8s.io images import calico.tar.gz</span>
<span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># ctr -n=k8s.io images import calico.tar.gz </span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># ctr -n=k8s.io images import calico.tar.gz </span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>

<p>上传calico.yaml到master1上，使用yaml文件安装calico 网络插件</p>
<blockquote>
<p>注：在线下载配置文件地址是： <a target="_blank" rel="noopener" href="https://docs.projectcalico.org/manifests/calico.yaml">https://docs.projectcalico.org/manifests/calico.yaml</a></p>
</blockquote>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl apply -f calico.yaml </span>
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl get nodes</span>
NAME      STATUS   ROLES           AGE     VERSION
master1   Ready    control-plane   35m     v1.25.0
node1     Ready    work            9m2s    v1.25.0
node2     Ready    work            7m14s   v1.25.0
<span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl get pods -n kube-system</span>
NAME                                       READY   STATUS    RESTARTS   AGE
calico-kube-controllers-6744f6b6d5-nnblc   <span class="token number">1</span>/1     Running   <span class="token number">0</span>          106s
calico-node-22g7q                          <span class="token number">1</span>/1     Running   <span class="token number">0</span>          106s
calico-node-fvp5g                          <span class="token number">1</span>/1     Running   <span class="token number">0</span>          106s
calico-node-wqwwg                          <span class="token number">1</span>/1     Running   <span class="token number">0</span>          106s
coredns-7f8cbcb969-2r662                   <span class="token number">1</span>/1     Running   <span class="token number">0</span>          37m
coredns-7f8cbcb969-kpgsv                   <span class="token number">1</span>/1     Running   <span class="token number">0</span>          37m
etcd-master1                               <span class="token number">1</span>/1     Running   <span class="token number">0</span>          37m
kube-apiserver-master1                     <span class="token number">1</span>/1     Running   <span class="token number">0</span>          37m
kube-controller-manager-master1            <span class="token number">1</span>/1     Running   <span class="token number">0</span>          37m
kube-proxy-7vjk5                           <span class="token number">1</span>/1     Running   <span class="token number">0</span>          8m42s
kube-proxy-mfpcg                           <span class="token number">1</span>/1     Running   <span class="token number">0</span>          37m
kube-proxy-pv87q                           <span class="token number">1</span>/1     Running   <span class="token number">0</span>          10m
kube-scheduler-master1                     <span class="token number">1</span>/1     Running   <span class="token number">0</span>          37m<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h4 id="2-5-calico网络插件配置文件说明"><a href="#2-5-calico网络插件配置文件说明" class="headerlink" title="2.5 calico网络插件配置文件说明"></a>2.5 calico网络插件配置文件说明</h4><pre class="line-numbers language-yaml" data-language="yaml"><code class="language-yaml"><span class="token comment"># Cluster type to identify the deployment type</span>
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> CLUSTER_TYPE
  <span class="token key atrule">value</span><span class="token punctuation">:</span> <span class="token string">"k8s,bgp"</span>
<span class="token comment"># Auto-detect the BGP IP address.</span>
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> IP
  <span class="token key atrule">value</span><span class="token punctuation">:</span> <span class="token string">"autodetect"</span>
<span class="token comment"># Enable IPIP</span>
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> CALICO_IPV4POOL_IPIP
  <span class="token key atrule">value</span><span class="token punctuation">:</span> <span class="token string">"Always"</span>
<span class="token comment"># Enable or Disable VXLAN on the default IP pool.</span>
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> CALICO_IPV4POOL_VXLAN
  <span class="token key atrule">value</span><span class="token punctuation">:</span> <span class="token string">"Never"</span>
<span class="token comment"># Set MTU for tunnel device used if ipip is enabled</span>
<span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> FELIX_IPINIPMTU
  valueFrom<span class="token punctuation">:</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>calico-node服务的主要参数如下:<br><strong>CALICO_IPV4POOL_IPIP：</strong> 是否启用IPIP模式。启用IPIP模式时，Calico将在Node上创建一个名为tunl0的虚拟隧道。IP Pool可以使用两种模式：BGP或IPIP。使用IPIP模式时，设置CALICO_IPV4POOL_IPIP&#x3D;”Always”，不使用IPIP模式时，设置CALICO_IPV4POOL_IPIP&#x3D;”Off”，此时将使用BGP模式。</p>
<p><strong>IP_AUTODETECTION_METHOD</strong>：获取Node IP地址的方式，默认使用第1个网络接口的IP地址，对于安装了多块网卡的Node，可以使用正则表达式选择正确的网卡，例如”interface&#x3D;eth.*”  表示选择名称以eth开头的网卡的IP地址。</p>
<pre class="line-numbers language-yaml" data-language="yaml"><code class="language-yaml"><span class="token punctuation">-</span>  name<span class="token punctuation">:</span> IP_AUTODETECTION_METHOD
  value<span class="token punctuation">:</span> <span class="token string">"interface=ens33"</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>扩展：calico的IPIP模式和BGP模式对比分析</p>
<p><strong>1）IPIP</strong><br>把一个IP数据包又套在一个IP包里，即把IP层封装到IP层的一个 tunnel，它的作用其实基本上就相当于一个基于IP层的网桥，一般来说，普通的网桥是基于mac层的，根本不需要IP，而这个ipip则是通过两端的路由做一个tunnel，把两个本来不通的网络通过点对点连接起来；</p>
<p>calico以ipip模式部署完毕后，node上会有一个tunl0的网卡设备，这是ipip做隧道封装用的,也是一种overlay模式的网络。当我们把节点下线，calico容器都停止后，这个设备依然还在，执行 rmmodipip命令可以将它删除。</p>
<p><strong>2）BGP</strong><br>BGP模式直接使用物理机作为虚拟路由路（vRouter），不再创建额外的tunnel</p>
<p>边界网关协议（BorderGateway Protocol, BGP）是互联网上一个核心的去中心化的自治路由协议。它通过维护IP路由表或‘前缀’表来实现自治系统（AS）之间的可达性，属于矢量路由协议。BGP不使用传统的内部网关协议（IGP）的指标，而是基于路径、网络策略或规则集来决定路由。因此，它更适合被称为矢量性协议，而不是路由协议，通俗的说就是将接入到机房的多条线路（如电信、联通、移动等）融合为一体，实现多线单IP；</p>
<p>BGP 机房的优点：服务器只需要设置一个IP地址，最佳访问路由是由网络上的骨干路由器根据路由跳数与其它技术指标来确定的，不会占用服务器的任何系统；</p>
<p>官方提供的calico.yaml模板里，默认打开了ip-ip功能，该功能会在node上创建一个设备tunl0，容器的网络数据会经过该设备被封装一个ip头再转发。这里，calico.yaml中通过修改calico-node的环境变量：CALICO_IPV4POOL_IPIP来实现ipip功能的开关：默认是Always，表示开启；Off表示关闭ipip。</p>
<pre class="line-numbers language-yaml" data-language="yaml"><code class="language-yaml"><span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span>  CLUSTER_TYPE
              value<span class="token punctuation">:</span> <span class="token string">"k8s,bgp"</span>
            <span class="token comment"># Auto-detect the BGP IP address.</span>
            <span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> IP
              value<span class="token punctuation">:</span> <span class="token string">"autodetect"</span>
            <span class="token comment"># Enable IPIP</span>
            <span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> CALICO_IPV4POOL_IPIP
              value<span class="token punctuation">:</span> <span class="token string">"Always"</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<p>总结：<br>calico BGP通信是基于TCP协议的，所以只要节点间三层互通即可完成，即三层互通的环境bird就能生成与邻居有关的路由。但是这些路由和flannel host-gateway模式一样，需要二层互通才能访问的通，因此如果在实际环境中配置了BGP模式生成了路由但是不同节点间pod访问不通，可能需要再确认下节点间是否二层互通。</p>
<p>为了解决节点间二层不通场景下的跨节点通信问题，calico也有自己的解决方案——IPIP模式</p>
<h4 id="2-6-测试"><a href="#2-6-测试" class="headerlink" title="2.6 测试"></a>2.6 测试</h4><p>在k8s上创建pod是否可以正常访问网络<br>把busybox-1-28.tar.gz上传到node1和node2节点，手动解压</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@node1 ~<span class="token punctuation">]</span><span class="token comment"># ctr -n k8s.io images import busybox-1-28.tar.gz</span>
<span class="token punctuation">[</span>root@node2 ~<span class="token punctuation">]</span><span class="token comment"># ctr -n k8s.io images import busybox-1-28.tar.gz</span><span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<p>在master1节点运行，并测试</p>
<pre class="line-numbers language-bash" data-language="bash"><code class="language-bash"><span class="token punctuation">[</span>root@master1 ~<span class="token punctuation">]</span><span class="token comment"># kubectl run busybox --image docker.io/library/busybox:1.28  --image-pull-policy=IfNotPresent --restart=Never --rm -it busybox -- sh</span>
If you don't see a <span class="token builtin class-name">command</span> prompt, try pressing enter.
/ <span class="token comment"># ping www.baidu.com</span>
PING www.baidu.com <span class="token punctuation">(</span><span class="token number">110.242</span>.68.4<span class="token punctuation">)</span>: <span class="token number">56</span> data bytes
<span class="token number">64</span> bytes from <span class="token number">110.242</span>.68.4: <span class="token assign-left variable">seq</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">ttl</span><span class="token operator">=</span><span class="token number">127</span> <span class="token assign-left variable">time</span><span class="token operator">=</span><span class="token number">13.429</span> ms
<span class="token number">64</span> bytes from <span class="token number">110.242</span>.68.4: <span class="token assign-left variable">seq</span><span class="token operator">=</span><span class="token number">1</span> <span class="token assign-left variable">ttl</span><span class="token operator">=</span><span class="token number">127</span> <span class="token assign-left variable">time</span><span class="token operator">=</span><span class="token number">11.507</span> ms
^C
--- www.baidu.com <span class="token function">ping</span> statistics ---
<span class="token number">2</span> packets transmitted, <span class="token number">2</span> packets received, <span class="token number">0</span>% packet loss
round-trip min/avg/max <span class="token operator">=</span> <span class="token number">11.507</span>/12.468/13.429 ms
<span class="token comment"># 通过上面可以看可以访问网络，说明calico网络插件已经被正常安装了</span>
/ <span class="token comment"># nslookup kubernetes.default.svc.cluster.local</span>
Server:    <span class="token number">10.96</span>.0.10
Address <span class="token number">1</span>: <span class="token number">10.96</span>.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kubernetes.default.svc.cluster.local
Address <span class="token number">1</span>: <span class="token number">10.96</span>.0.1 kubernetes.default.svc.cluster.local
/ <span class="token comment"># exit</span>
pod <span class="token string">"busybox"</span> deleted
You have new mail <span class="token keyword">in</span> /var/spool/mail/root
<span class="token comment"># 10.96.0.10 就是我们coreDNS的clusterIP，说明coreDNS配置好了。</span>
解析内部Service的名称，是通过coreDNS去解析的<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<h3 id="三、ctr和crictl的区别"><a href="#三、ctr和crictl的区别" class="headerlink" title="三、ctr和crictl的区别"></a>三、ctr和crictl的区别</h3><p>背景：在部署k8s的过程中，经常要对镜像进行操作（拉取、删除、查看等）<br>问题：使用过程中会发现ctr和crictl有很多相同功能，也有些不同，那区别到底在哪里？<br>说明：</p>
<ol>
<li>ctr是containerd自带的CLI命令行工具，crictl是k8s中CRI（容器运行时接口）的客户端，k8s使用该客户端和containerd进行交互；</li>
<li>ctr和crictl命令具体区别如下，也可以–help查看。crictl缺少对具体镜像的管理能力，可能是k8s层面镜像管理可以由用户自行控制，能配置pod里面容器的统一镜像仓库，镜像的管理可以有habor等插件进行处理。</li>
</ol>
<p><img src="https://gitee.com/miaohsukang/myblog/raw/master/img/image-20221103133432356.png"></p>

                
            </div>
            <hr/>

            

    <div class="reprint" id="reprint-statement">
        
            <div class="reprint__author">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-user">
                        文章作者:
                    </i>
                </span>
                <span class="reprint-info">
                    <a href="/about" rel="external nofollow noreferrer">Miaohsukang</a>
                </span>
            </div>
            <div class="reprint__type">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-link">
                        文章链接:
                    </i>
                </span>
                <span class="reprint-info">
                    <a href="https://miaohsukang.gitee.io/2023/02/12/k8s-bi-ji/di-1-zhang-an-zhuang-k8s1-25-gao-ke-yong-ji-qun-md/">https://miaohsukang.gitee.io/2023/02/12/k8s-bi-ji/di-1-zhang-an-zhuang-k8s1-25-gao-ke-yong-ji-qun-md/</a>
                </span>
            </div>
            <div class="reprint__notice">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-copyright">
                        版权声明:
                    </i>
                </span>
                <span class="reprint-info">
                    本博客所有文章除特別声明外，均采用
                    <a href="https://creativecommons.org/licenses/by/4.0/deed.zh" rel="external nofollow noreferrer" target="_blank">CC BY 4.0</a>
                    许可协议。转载请注明来源
                    <a href="/about" target="_blank">Miaohsukang</a>
                    !
                </span>
            </div>
        
    </div>

    <script async defer>
      document.addEventListener("copy", function (e) {
        let toastHTML = '<span>复制成功，请遵循本文的转载规则</span><button class="btn-flat toast-action" onclick="navToReprintStatement()" style="font-size: smaller">查看</a>';
        M.toast({html: toastHTML})
      });

      function navToReprintStatement() {
        $("html, body").animate({scrollTop: $("#reprint-statement").offset().top - 80}, 800);
      }
    </script>



            <div class="tag_share" style="display: block;">
                <div class="post-meta__tag-list" style="display: inline-block;">
                    
                        <div class="article-tag">
                            
                                <a href="/tags/k8s%E9%9B%86%E7%BE%A4%E5%AE%89%E8%A3%85/">
                                    <span class="chip bg-color">k8s集群安装</span>
                                </a>
                            
                        </div>
                    
                </div>
                <div class="post_share" style="zoom: 80%; width: fit-content; display: inline-block; float: right; margin: -0.15rem 0;">
                    <link rel="stylesheet" type="text/css" href="/libs/share/css/share.min.css">
<div id="article-share">

    
    <div class="social-share" data-sites="twitter,facebook,google,qq,qzone,wechat,weibo,douban,linkedin" data-wechat-qrcode-helper="<p>微信扫一扫即可分享！</p>"></div>
    <script src="/libs/share/js/social-share.min.js"></script>
    

    

</div>

                </div>
            </div>
            
                <div id="reward">
    <a href="#rewardModal" class="reward-link modal-trigger btn-floating btn-medium waves-effect waves-light red">赏</a>

    <!-- Modal Structure -->
    <div id="rewardModal" class="modal">
        <div class="modal-content">
            <a class="close modal-close"><i class="fas fa-times"></i></a>
            <h4 class="reward-title">你的赏识是我前进的动力</h4>
            <div class="reward-content">
                <div class="reward-tabs">
                    <ul class="tabs row">
                        <li class="tab col s6 alipay-tab waves-effect waves-light"><a href="#alipay">支付宝</a></li>
                        <li class="tab col s6 wechat-tab waves-effect waves-light"><a href="#wechat">微 信</a></li>
                    </ul>
                    <div id="alipay">
                        <img src="/medias/reward/alipay.jpg" class="reward-img" alt="支付宝打赏二维码">
                    </div>
                    <div id="wechat">
                        <img src="/medias/reward/wechat.png" class="reward-img" alt="微信打赏二维码">
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>

<script>
    $(function () {
        $('.tabs').tabs();
    });
</script>

            
        </div>
    </div>

    

    

    

    

    

    

    

    

    

<article id="prenext-posts" class="prev-next articles">
    <div class="row article-row">
        
        <div class="article col s12 m6" data-aos="fade-up">
            <div class="article-badge left-badge text-color">
                <i class="fas fa-chevron-left"></i>&nbsp;上一篇</div>
            <div class="card">
                <a href="/2023/02/12/k8s-bi-ji/di-2-zhang-pod-gai-shu-md/">
                    <div class="card-image">
                        
                        <img src="https://www.zutuanxue.com:8000/static/media/images/2020/10/10/1602326898782.png" class="responsive-img" alt="第2章-pod概述.md">
                        
                        <span class="card-title">第2章-pod概述.md</span>
                    </div>
                </a>
                <div class="card-content article-content">
                    <div class="summary block-with-text">
                        
                            
                        
                    </div>
                    <div class="publish-info">
                        <span class="publish-date">
                            <i class="far fa-clock fa-fw icon-date"></i>2023-02-12
                        </span>
                        <span class="publish-author">
                            
                            <i class="fas fa-bookmark fa-fw icon-category"></i>
                            
                            <a href="/categories/K8S%E7%AC%94%E8%AE%B0/" class="post-category">
                                    K8S笔记
                                </a>
                            
                            
                        </span>
                    </div>
                </div>
                
                <div class="card-action article-tags">
                    
                    <a href="/tags/pod/">
                        <span class="chip bg-color">pod</span>
                    </a>
                    
                </div>
                
            </div>
        </div>
        
        
        <div class="article col s12 m6" data-aos="fade-up">
            <div class="article-badge right-badge text-color">
                下一篇&nbsp;<i class="fas fa-chevron-right"></i>
            </div>
            <div class="card">
                <a href="/2023/02/12/docker/an-zhuang-docker/">
                    <div class="card-image">
                        
                        <img src="https://www.lixuan222.com/wp-content/uploads/2020/07/docker.png" class="responsive-img" alt="安装docker">
                        
                        <span class="card-title">安装docker</span>
                    </div>
                </a>
                <div class="card-content article-content">
                    <div class="summary block-with-text">
                        
                            
                        
                    </div>
                    <div class="publish-info">
                            <span class="publish-date">
                                <i class="far fa-clock fa-fw icon-date"></i>2023-02-12
                            </span>
                        <span class="publish-author">
                            
                            <i class="fas fa-bookmark fa-fw icon-category"></i>
                            
                            <a href="/categories/Docker/" class="post-category">
                                    Docker
                                </a>
                            
                            
                        </span>
                    </div>
                </div>
                
                <div class="card-action article-tags">
                    
                    <a href="/tags/docker/">
                        <span class="chip bg-color">docker</span>
                    </a>
                    
                </div>
                
            </div>
        </div>
        
    </div>
</article>

</div>



<!-- 代码块功能依赖 -->
<script type="text/javascript" src="/libs/codeBlock/codeBlockFuction.js"></script>


  <!-- 是否加载使用自带的 prismjs. -->
  <script type="text/javascript" src="/libs/prism/prism.min.js"></script>


<!-- 代码语言 -->

<script type="text/javascript" src="/libs/codeBlock/codeLang.js"></script>


<!-- 代码块复制 -->

<script type="text/javascript" src="/libs/codeBlock/codeCopy.js"></script>


<!-- 代码块收缩 -->

<script type="text/javascript" src="/libs/codeBlock/codeShrink.js"></script>



    </div>
    <div id="toc-aside" class="expanded col l3 hide-on-med-and-down">
        <div class="toc-widget card" style="background-color: white;">
            <div class="toc-title"><i class="far fa-list-alt"></i>&nbsp;&nbsp;目录</div>
            <div id="toc-content"></div>
        </div>
    </div>
</div>

<!-- TOC 悬浮按钮. -->


<script src="/libs/tocbot/tocbot.min.js"></script>
<script>
    $(function () {
        tocbot.init({
            tocSelector: '#toc-content',
            contentSelector: '#articleContent',
            headingsOffset: -($(window).height() * 0.4 - 45),
            collapseDepth: Number('0'),
            headingSelector: 'h2, h3, h4'
        });

        // Set scroll toc fixed.
        let tocHeight = parseInt($(window).height() * 0.4 - 64);
        let $tocWidget = $('.toc-widget');
        $(window).scroll(function () {
            let scroll = $(window).scrollTop();
            /* add post toc fixed. */
            if (scroll > tocHeight) {
                $tocWidget.addClass('toc-fixed');
            } else {
                $tocWidget.removeClass('toc-fixed');
            }
        });

        
    });
</script>

    

</main>




    <footer class="page-footer bg-color">
    
        <link rel="stylesheet" href="/libs/aplayer/APlayer.min.css">
<style>
    .aplayer .aplayer-lrc p {
        
        display: none;
        
        font-size: 12px;
        font-weight: 700;
        line-height: 16px !important;
    }

    .aplayer .aplayer-lrc p.aplayer-lrc-current {
        
        display: none;
        
        font-size: 15px;
        color: #42b983;
    }

    
    .aplayer.aplayer-fixed.aplayer-narrow .aplayer-body {
        left: -66px !important;
    }

    .aplayer.aplayer-fixed.aplayer-narrow .aplayer-body:hover {
        left: 0px !important;
    }

    
</style>
<div class="">
    
    <div class="row">
        <meting-js class="col l8 offset-l2 m10 offset-m1 s12"
                   server="netease"
                   type="playlist"
                   id="503838841"
                   fixed='true'
                   autoplay='false'
                   theme='#42b983'
                   loop='all'
                   order='random'
                   preload='auto'
                   volume='0.7'
                   list-folded='true'
        >
        </meting-js>
    </div>
</div>

<script src="/libs/aplayer/APlayer.min.js"></script>
<script src="/libs/aplayer/Meting.min.js"></script>

    

    <div class="container row center-align"
         style="margin-bottom: 0px !important;">
        <div class="col s12 m8 l8 copy-right">
            Copyright&nbsp;&copy;
            
                <span id="year">2023</span>
            
            <a href="/about" target="_blank">Miaohsukang</a>
            |&nbsp;Powered by&nbsp;<a href="https://hexo.io/" target="_blank">Hexo</a>
            |&nbsp;Theme&nbsp;<a href="https://miaohsukang.gitee.io" target="_blank">Matery</a>
            
            <br>
            
            
            
                
            
            
                <span id="busuanzi_container_site_pv">
                &nbsp;|&nbsp;<i class="far fa-eye"></i>&nbsp;总访问量:&nbsp;
                    <span id="busuanzi_value_site_pv" class="white-color"></span>
            </span>
            
            
                <span id="busuanzi_container_site_uv">
                &nbsp;|&nbsp;<i class="fas fa-users"></i>&nbsp;总访问人数:&nbsp;
                    <span id="busuanzi_value_site_uv" class="white-color"></span>
            </span>
            
            <br>

            <!-- 运行天数提醒. -->
            
            <br>
            
        </div>
        <div class="col s12 m4 l4 social-link social-statis">
    <a href="https://miaohsukang.gitee.io" class="tooltipped" target="_blank" data-tooltip="访问我的GitHub" data-position="top" data-delay="50">
        <i class="fab fa-github"></i>
    </a>



    <a href="mailto:12419315+miaohsukang@user.noreply.gitee.com" class="tooltipped" target="_blank" data-tooltip="邮件联系我" data-position="top" data-delay="50">
        <i class="fas fa-envelope-open"></i>
    </a>



    <a href="tencent://AddContact/?fromId=50&fromSubId=1&subcmd=all&uin=258603904" class="tooltipped" target="_blank" data-tooltip="QQ联系我: 258603904" data-position="top" data-delay="50">
        <i class="fab fa-qq"></i>
    </a>

</div>
    </div>
</footer>

<div class="progress-bar"></div>


    <!-- 搜索遮罩框 -->
<div id="searchModal" class="modal">
    <div class="modal-content">
        <div class="search-header">
            <span class="title"><i class="fas fa-search"></i>&nbsp;&nbsp;搜索</span>
            <input type="search" id="searchInput" name="s" placeholder="请输入搜索的关键字"
                   class="search-input">
        </div>
        <div id="searchResult"></div>
    </div>
</div>

<script type="text/javascript">
$(function () {
    var searchFunc = function (path, search_id, content_id) {
        'use strict';
        $.ajax({
            url: path,
            dataType: "xml",
            success: function (xmlResponse) {
                // get the contents from search data
                var datas = $("entry", xmlResponse).map(function () {
                    return {
                        title: $("title", this).text(),
                        content: $("content", this).text(),
                        url: $("url", this).text()
                    };
                }).get();
                var $input = document.getElementById(search_id);
                var $resultContent = document.getElementById(content_id);
                $input.addEventListener('input', function () {
                    var str = '<ul class=\"search-result-list\">';
                    var keywords = this.value.trim().toLowerCase().split(/[\s\-]+/);
                    $resultContent.innerHTML = "";
                    if (this.value.trim().length <= 0) {
                        return;
                    }
                    // perform local searching
                    datas.forEach(function (data) {
                        var isMatch = true;
                        var data_title = data.title.trim().toLowerCase();
                        var data_content = data.content.trim().replace(/<[^>]+>/g, "").toLowerCase();
                        var data_url = data.url;
                        data_url = data_url.indexOf('/') === 0 ? data.url : '/' + data_url;
                        var index_title = -1;
                        var index_content = -1;
                        var first_occur = -1;
                        // only match artiles with not empty titles and contents
                        if (data_title !== '' && data_content !== '') {
                            keywords.forEach(function (keyword, i) {
                                index_title = data_title.indexOf(keyword);
                                index_content = data_content.indexOf(keyword);
                                if (index_title < 0 && index_content < 0) {
                                    isMatch = false;
                                } else {
                                    if (index_content < 0) {
                                        index_content = 0;
                                    }
                                    if (i === 0) {
                                        first_occur = index_content;
                                    }
                                }
                            });
                        }
                        // show search results
                        if (isMatch) {
                            str += "<li><a href='" + data_url + "' class='search-result-title'>" + data_title + "</a>";
                            var content = data.content.trim().replace(/<[^>]+>/g, "");
                            if (first_occur >= 0) {
                                // cut out 100 characters
                                var start = first_occur - 20;
                                var end = first_occur + 80;
                                if (start < 0) {
                                    start = 0;
                                }
                                if (start === 0) {
                                    end = 100;
                                }
                                if (end > content.length) {
                                    end = content.length;
                                }
                                var match_content = content.substr(start, end);
                                // highlight all keywords
                                keywords.forEach(function (keyword) {
                                    var regS = new RegExp(keyword, "gi");
                                    match_content = match_content.replace(regS, "<em class=\"search-keyword\">" + keyword + "</em>");
                                });

                                str += "<p class=\"search-result\">" + match_content + "...</p>"
                            }
                            str += "</li>";
                        }
                    });
                    str += "</ul>";
                    $resultContent.innerHTML = str;
                });
            }
        });
    };

    searchFunc('/search.xml', 'searchInput', 'searchResult');
});
</script>

    <!-- 白天和黑夜主题 -->
<div class="stars-con">
    <div id="stars"></div>
    <div id="stars2"></div>
    <div id="stars3"></div>  
</div>

<script>
    function switchNightMode() {
        $('<div class="Cuteen_DarkSky"><div class="Cuteen_DarkPlanet"></div></div>').appendTo($('body')),
        setTimeout(function () {
            $('body').hasClass('DarkMode') 
            ? ($('body').removeClass('DarkMode'), localStorage.setItem('isDark', '0'), $('#sum-moon-icon').removeClass("fa-sun").addClass('fa-moon')) 
            : ($('body').addClass('DarkMode'), localStorage.setItem('isDark', '1'), $('#sum-moon-icon').addClass("fa-sun").removeClass('fa-moon')),
            
            setTimeout(function () {
            $('.Cuteen_DarkSky').fadeOut(1e3, function () {
                $(this).remove()
            })
            }, 2e3)
        })
    }
</script>

    <!-- 回到顶部按钮 -->
<div id="backTop" class="top-scroll">
    <a class="btn-floating btn-large waves-effect waves-light" href="#!">
        <i class="fas fa-arrow-up"></i>
    </a>
</div>


    <script src="/libs/materialize/materialize.min.js"></script>
    <script src="/libs/masonry/masonry.pkgd.min.js"></script>
    <script src="/libs/aos/aos.js"></script>
    <script src="/libs/scrollprogress/scrollProgress.min.js"></script>
    <script src="/libs/lightGallery/js/lightgallery-all.min.js"></script>
    <script src="/js/matery.js"></script>

    

    
    
    

    <!-- 雪花特效 -->
    

    <!-- 鼠标星星特效 -->
    

     
        <script src="https://ssl.captcha.qq.com/TCaptcha.js"></script>
        <script src="/libs/others/TencentCaptcha.js"></script>
        <button id="TencentCaptcha" data-appid="xxxxxxxxxx" data-cbfn="callback" type="button" hidden></button>
    

    <!-- Baidu Analytics -->

    <!-- Baidu Push -->

<script>
    (function () {
        var bp = document.createElement('script');
        var curProtocol = window.location.protocol.split(':')[0];
        if (curProtocol === 'https') {
            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
        } else {
            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
        }
        var s = document.getElementsByTagName("script")[0];
        s.parentNode.insertBefore(bp, s);
    })();
</script>

    
    <script src="/libs/others/clicklove.js" async="async"></script>
    
    
    <script async src="/libs/others/busuanzi.pure.mini.js"></script>
    

    

    

    <!--腾讯兔小巢-->
    
    

    

    

    
    <script src="/libs/instantpage/instantpage.js" type="module"></script>
    

</body>

</html>
